class GPT2MedicalQADataSet_txt_ntrain(Dataset):
    def __init__(self,file_paths,nraws,shuffle , tokenizer, max_len, data_dir, data_set_name , is_overwrite=False):
        """
        初始化函数
        Args:

            file_path： txt 文件路径  # file_raws ,f_n = 10  # 文件总数
            tokenizer: 分词器
            max_len: 数据的最大长度
            data_dir: 保存缓存文件的路径
            data_set_name: 数据集名字
            path_file: 原始数据文件
            is_overwrite: 是否重新生成缓存文件
        """
        self.node_files = 200 # 文件在节点的数量
        self.len_idx = 54
        # f_n = 873 # 文件总数        
        # self.dload_n = 4
        # 文件描述
        # self.fpi = 0 # 第一个文件

        # self.f_n =  f_n    # 文件总数         
        # 在一个文件中 读取的数据数量  全部读取
        # self.nraws = nraws

        self.tokenizer = tokenizer
        # content_id和title_id分别对应新闻的正文和标题，为了在模型中区分的更明显
        self.content_id = self.tokenizer.convert_tokens_to_ids("[Content]")
        self.title_id = self.tokenizer.convert_tokens_to_ids("[Title]")
        # space_id表示空格标记，由于一些标题中带有空格，如果直接使用tokenizer进行分词，会导致空格消失，会显得标题很奇怪
        # 但是又不方便同一替换成任意一个标点，因此将其用[Space]替换。
        self.space_id = self.tokenizer.convert_tokens_to_ids("[Space]")
        self.max_len = max_len  
        self.shuffle = shuffle
        # self.save = open('file.txt','a')

    def initial(self): 
        self.fpi_add =  eval(open('fpi.txt').readline())-1  
        self.samples = list()
        self.get_fpi(self.fpi_add )
        self.samples =self.samples_next  
        self.samples_next =[]
        # file_raws  = len( self.samples)    #每一个文件的行数
        # self.file_raws = file_raws * self.f_n        
        # print("train  file_raws" , file_raws , self.f_n ,   self.file_raws )    
        
        # put nraw samples into memory
        self.current_sample_num = len(self.samples)
        self.index = list(range(self.current_sample_num))
        # print(self.index)
        if self.shuffle:
            random.shuffle(self.samples)
    def __len__(self):
        return 47142 #self.file_raws #len(self.data_set)
    def __getitem__(self,idx):
        a =time.time()
        # idx_n = self.index[0]
        # print(idx_n,idx,self.current_sample_num)        
        data = self.samples[self.index[0]]
        # print(idx,len(self.index), data['input_ids'][-12:-4])
        self.index = self.index[1:]
        self.current_sample_num-=1
        # print(self.current_sample_num ,time.time() )
        if self.current_sample_num==0:
            print("_0_",time.time())
            # self.fpi =self.fpi+1  #0  1
            # self.get_fpi(int(idx /self.len_idx) )  
            self.samples =self.samples_next             
            self.current_sample_num = len(self.samples_next)
            self.samples_next =[]
            self.index = list(range(self.current_sample_num))
            if self.shuffle:
                random.shuffle(self.samples)
            #print("_0_",time.time())
        if self.current_sample_num == 10  :
            #print("_5_s",time.time(), len(self.samples_next))
            t = threading.Thread(target=self.get_fpi,args=(int(idx /self.len_idx),)  )
            # self.get_fpi(int(idx /self.len_idx) ) #给 samples_next
            t.start()
            # t.join()
            # print(a,time.time())
            print("_5_e",time.time(),len(self.samples_next))
        # print("_return_",time.time())
        # print("gen__", self.current_sample_num, len(self.samples_next),time.time()-a, time.time() ) 
        # self.save.write('\ng_\t'+str( time.time()-a)  )
        return data
    def get_fpi(self,fpi):
        # python http
        #url = 'http://10.7.0.100:31672/train_'+str( fpi)+'.txt'        
        # fastdfs
        '''        
        文件偏移  0-5   0
                 6-11   1 
        判断文件在哪一个服务器
        '''   
        a =time.time()
        # fpi  = self.fpi_add + fpi*6  #6 是 节点的数量  
        # iii = int(  fpi/10000 )   #4 是文件在节点的数量 
        # path = "/home/ubuntu/disk/fastfs_gpt2data/files/default/20210507/train_1350_10/"
        # fpp  = path +str( int(fpi/100)) +"/"+fname
        # print(fpp)
        # self.samples = [ eval(a) for a in  open(fpp).readlines()[:1350] ] 
        iii = int(  fpi /self.node_files )   #4 是文件在节点的数量  
        ips = ['192.168.3.228','192.168.3.90','192.168.3.216','192.168.3.122','192.168.3.21','192.168.3.31']
        url  = "http://"+ips[iii]+":8080/group1/default/20210507/train_{len_idx}_10/{i}/{j}/{f1}.json?name={f2}.json&download=1".format(
            len_idx=self.len_idx,i=int(fpi/10000),j=int(fpi%10000/100) ,f1= int(fpi%10000%100),f2= int(fpi%10000%100))
        
        # ips = ['192.168.3.228','192.168.3.90','192.168.3.216','192.168.3.122','192.168.3.21','192.168.3.31']
        # url  = "http://"+ips[iii]+":8080/group1/default/20210507/train_{len_idx}_10/{ml}/{fn1}?name={fn2}&download=1".format(
        #     len_idx=self.len_idx,ml=str( int(fpi/100)),fn1=fname,fn2=fname)
        # url  = url  +str( int(fpi/100)) +"/"+fname+"?name="+fname+"&download=1"
        # print(url )
        self.samples_next =json.loads(requests.get(url).text)
        # self.samples_next = [ eval(a) for a in requests.get(url).content.split(b'\n')[:self.len_idx] ] # 这个比下面的快一点
        if self.shuffle:
            random.shuffle(self.samples_next)
        # print(   "\ngf",time.time()-a,a,time.time() )
        # self.save.write('\ngf\t'+str( time.time()-a) )

